@data-fair/dev-server 1.13.1 → 2.0.1

package/package.json

@@ -1,19 +1,26 @@
 {
   "name": "@data-fair/dev-server",
-  "version": "1.13.1",
+  "version": "2.0.1",
   "description": "A development server for optimal development experience of data-fair applications.",
-  "main": "server/index.js",
+  "main": "src/index.js",
   "bin": {
-    "df-dev-server": "server/index.js"
+    "df-dev-server": "src/index.js"
   },
+  "type": "module",
   "scripts": {
-    "dev": "NODE_ENV=development nodemon server",
-    "dev-src": "http-server test-apps/minimal --port=3000 --cors --silent",
-    "dev-vue3": "APP_URL=http://localhost:3000/app/ APP_SCRIPT=dev-src-vue3 NODE_ENV=development nodemon server",
-    "dev-src-vue3": "cd test-apps/vue3 && npm run dev",
-    "prepare": "npm run lint && nuxt generate",
-    "lint": "eslint --ext .js,.vue ."
+    "build": "tsc",
+    "dev": "NODE_ENV=development node --watch src/index.js",
+    "dev-test-app-minimal": "http-server test-apps/minimal --port=3000 --cors --silent",
+    "dev-test-app-vue3": "cd test-apps/vue3 && npm run dev",
+    "dev-zellij": "zellij --layout .zellij.kdl",
+    "prepublishOnly": "npm run lint && npm run build &&npm -w ui run build",
+    "lint": "eslint . && npm -w ui run lint",
+    "lint-fix": "eslint --fix . && npm -w ui run lint-fix",
+    "build-types": "df-build-types ."
   },
+  "workspaces": [
+    "ui"
+  ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/koumoul-dev/df-dev-server.git"
@@ -22,66 +29,44 @@
     "data-fair",
     "dev"
   ],
-  "author": "Alban Mouton <alban.mouton@gmail.com>",
+  "author": "Alban Mouton <alban.mouton@koumoul.com>",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/koumoul-dev/df-dev-server/issues"
   },
   "homepage": "https://github.com/koumoul-dev/df-dev-server#readme",
-  "eslintConfig": {
-    "root": true,
-    "env": {
-      "node": true
-    },
-    "extends": [
-      "plugin:vue/essential",
-      "eslint:recommended",
-      "standard",
-      "plugin:vue/recommended"
-    ],
-    "rules": {
-      "vue/no-v-html": "off",
-      "vue/multi-word-component-names": "off"
-    }
+  "devDependencies": {
+    "@data-fair/lib-types-builder": "^1.8.2",
+    "@types/config": "^3.3.5",
+    "@types/cors": "^2.8.19",
+    "@types/debug": "^4.1.12",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.0.7",
+    "@types/ws": "^8.18.1",
+    "eslint": "^9.30.0",
+    "eslint-plugin-vue": "^10.2.0",
+    "eslint-plugin-vuetify": "^2.5.2",
+    "http-server": "^14.1.1",
+    "neostandard": "^0.12.1",
+    "typescript": "^5.8.3"
   },
   "dependencies": {
-    "@koumoul/nuxt-config-inject": "^0.4.4",
-    "@koumoul/vjsf": "^2.18.0",
-    "@koumoul/v-iframe": "^1.3.4",
-    "ajv": "^8.8.2",
-    "ajv-i18n": "^4.2.0",
-    "body-parser": "^1.19.1",
-    "chalk": "^3.0.0",
-    "config": "^3.3.6",
+    "@data-fair/lib-express": "^1.20.4",
+    "@parse5/tools": "^0.6.0",
+    "chalk": "^5.4.1",
+    "config": "^4.0.0",
     "cors": "^2.8.5",
-    "dot-prop": "^6.0.1",
-    "dotenv": "^10.0.0",
-    "escape-string-regexp": "^2.0.0",
-    "event-to-promise": "^0.8.0",
-    "express": "^4.17.2",
-    "fs-extra": "^10.0.0",
-    "html-extractor": "^0.2.2",
-    "http-proxy-middleware": "^2.0.1",
-    "iframe-resizer": "^4.3.2",
-    "parse5": "^6.0.1",
-    "tree-kill": "^1.2.2",
-    "ws": "^8.3.0"
+    "debug": "^4.4.1",
+    "dotenv": "^17.0.0",
+    "express": "^5.1.0",
+    "http-proxy-middleware": "^3.0.5",
+    "parse5": "^7.3.0",
+    "prom-client": "^15.1.3",
+    "ws": "^8.18.3"
   },
-  "devDependencies": {
-    "@babel/eslint-parser": "^7.16.0",
-    "@nuxtjs/axios": "^5.13.6",
-    "@nuxtjs/i18n": "^7.2.0",
-    "@nuxtjs/vuetify": "^1.12.3",
-    "eslint": "^7.12.1",
-    "eslint-config-standard": "^16.0.3",
-    "eslint-plugin-import": "^2.25.2",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^5.1.1",
-    "eslint-plugin-vue": "^8.0.3",
-    "http-server": "^0.12.3",
-    "nodemon": "^2.0.15",
-    "nuxt": "^2.15.8",
-    "reconnecting-websocket": "^4.4.0",
-    "vuetify": "^2.6.1"
-  }
+  "files": [
+    "./config/dist/**/*",
+    "./ui/dist/**/*",
+    "./src/**/*"
+  ]
 }

package/src/app.js

@@ -0,0 +1,273 @@
+// Express app for TaxMan own API and UI
+import config from './config.js';
+import uiConfig from './ui-config.js';
+import { WebSocket, WebSocketServer } from 'ws';
+import { createServer } from 'node:http';
+import express from 'express';
+import cors from 'cors';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import debugModule from 'debug';
+import { createProxyMiddleware, fixRequestBody } from 'http-proxy-middleware';
+import * as parse5 from 'parse5';
+import chalk from 'chalk';
+import { isElementNode, createTextNode, createElement } from '@parse5/tools';
+import escapeStringRegexp from 'escape-string-regexp';
+import eventPromise from '@data-fair/lib-utils/event-promise.js';
+import { createSpaMiddleware } from '@data-fair/lib-express/serve-spa.js';
+import { resolve } from 'node:path';
+const debug = debugModule('df-dev-server');
+const app = express();
+const server = createServer(app);
+// web socket server, used both by dev-server UI (opened on / path)
+// and by application (opened on /data-fair), in this case we transmit all messages to and from data-fair
+const wss = new WebSocketServer({ noServer: true });
+let devServerWS;
+let dataFairWS;
+server.on('upgrade', function upgrade(req, socket, head) {
+    wss.handleUpgrade(req, socket, head, (ws) => {
+        if (req.url === '/data-fair') {
+            if (dataFairWS)
+                dataFairWS.terminate();
+            dataFairWS = ws;
+            ws.on('message', (data) => {
+                if (dataFairOutputWS) {
+                    const message = JSON.parse(data);
+                    if (message.type === 'subscribe')
+                        message.apiKey = config.dataFair.apiKey;
+                    dataFairOutputWS.send(JSON.stringify(message));
+                }
+            });
+        }
+        else {
+            if (devServerWS)
+                devServerWS.terminate();
+            devServerWS = ws;
+        }
+        wss.emit('connection', ws, req);
+    });
+});
+const dfWsUrl = config.dataFair.url.replace('http://', 'ws://').replace('https://', 'wss://') + '/';
+const dataFairOutputWS = new WebSocket(dfWsUrl);
+dataFairOutputWS.on('open', () => dataFairOutputWS.send('connect to external websocket server ' + dfWsUrl));
+dataFairOutputWS.on('message', (data) => { if (dataFairWS)
+    dataFairWS.send(data.toString()); });
+dataFairOutputWS.on('error', (err) => console.error('failed to connect to external websocket server', dfWsUrl, err));
+app.use(cors());
+app.use(express.json());
+// very basic CRUD of config
+app.get('/config', (req, res, next) => {
+    const devConfig = existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {};
+    debug('read dev config', devConfig);
+    res.send(devConfig);
+});
+app.put('/config', (req, res, next) => {
+    debug('save dev config', req.body);
+    writeFileSync('.dev-config.json', JSON.stringify(req.body, null, 2));
+    res.send(req.body);
+});
+app.post('/config/error', (req, res) => {
+    console.log('Application sent an error', req.body);
+    if (devServerWS)
+        devServerWS.send(JSON.stringify({ type: 'app-error', data: req.body }));
+    else
+        console.error('no dev-server websocket open to send error to');
+    res.send();
+});
+// re-expose the application performing similar modifications to the body as data-fair
+const appUrl = new URL(config.app.url);
+const appPrefix = appUrl.pathname === '/' ? '' : appUrl.pathname;
+app.use('/app', createProxyMiddleware({
+    target: appUrl.origin,
+    secure: false,
+    changeOrigin: true,
+    selfHandleResponse: true, // so that the onProxyRes takes care of sending the response
+    on: {
+        proxyReq(proxyReq, req, res) {
+            proxyReq.path = appPrefix + proxyReq.path;
+            proxyReq.setHeader('Accept-Encoding', 'identity'); // disable compression
+        },
+        proxyRes(proxyRes, req, res) {
+            const configuration = existsSync('.dev-config.json') ? JSON.parse(readFileSync('.dev-config.json', 'utf8')) : {};
+            // console.log('inject config', configuration)
+            const dataBuffers = [];
+            proxyRes.on('data', (data) => { dataBuffers.push(data); });
+            proxyRes.on('end', async () => {
+                try {
+                    let output = Buffer.concat(dataBuffers).toString();
+                    if (output.includes('%APPLICATION%')) {
+                        const filledBody = output.replace(/%APPLICATION%/g, JSON.stringify({
+                            id: 'dev-application',
+                            title: 'Dev application',
+                            configuration,
+                            exposedUrl: `http://localhost:${config.port}/app`,
+                            href: `http://localhost:${config.port}/config`,
+                            apiUrl: `http://localhost:${config.port}/data-fair/api/v1`,
+                            wsUrl: `ws://localhost:${config.port}/data-fair`,
+                            owner: config.dataFair && config.dataFair.owner
+                        }));
+                        const document = parse5.parse(filledBody);
+                        const html = document.childNodes.filter(isElementNode).find(c => c.tagName === 'html');
+                        if (!html)
+                            throw new Error('HTML structure is broken, expect html, head and body elements');
+                        const headNode = html.childNodes.filter(isElementNode).find(c => c.tagName === 'head');
+                        const bodyNode = html.childNodes.filter(isElementNode).find(c => c.tagName === 'body');
+                        if (!headNode || !bodyNode)
+                            throw new Error('HTML structure is broken, expect html, head and body elements');
+                        // add a script to simulate instrumentation by capture service
+                        // @ts-ignore
+                        const query = req.query;
+                        if (query.thumbnail === 'true') {
+                            const script = createElement('script', { type: 'text/javascript' });
+                            script.childNodes.push(createTextNode(`
+              console.log('[capture] Simulate a screenshot capture context')
+              var triggerCalled = false
+              window.triggerCapture = function (animationSupported) {
+                triggerCalled = true
+                console.log('[capture] triggerCapture called')
+                if (animationSupported) {
+                  console.log('[capture] this application supports animated screenshots')
+                  var i = 0
+                  const interval = setInterval(function () {
+                    i++
+                    if (i === 1800) {
+                      console.error('[capture] stop after the maximum number of frames was attained')
+                      clearInterval(interval)
+                    }
+                    var stopped = window.animateCaptureFrame()
+                    if (stopped) {
+                      console.log('[capture] animation was stopped after ' + i + ' frames')
+                      clearInterval(interval)
+                    }
+                  }, 67)
+                  return true
+                } else {
+                  console.log('[capture] this application does not support animated screenshots')
+                }
+              }
+              setTimeout(function() {
+                if (!triggerCalled) {
+                  console.error('[capture] triggerCapture was not called after a 5s wait')
+                }
+              }, 5000)
+                              `));
+                            headNode.childNodes.push(script);
+                        }
+                        const meta = {};
+                        for (const node of headNode.childNodes.filter(isElementNode)) {
+                            if (node.tagName === 'meta') {
+                                const name = node.attrs.find(a => a.name === 'name')?.value;
+                                const content = node.attrs.find(a => a.name === 'content')?.value;
+                                if (name !== undefined && content !== undefined)
+                                    meta[name] = content;
+                            }
+                        }
+                        if (meta['df:sync-state'] === 'true' || meta['df:overflow'] === 'true') {
+                            bodyNode.childNodes.push(createElement('script', {
+                                type: 'text/javascript',
+                                src: 'https://cdn.jsdelivr.net/npm/@data-fair/frame@0.12/dist/v-iframe-compat/d-frame-content.min.js'
+                            }));
+                        }
+                        output = parse5.serialize(document);
+                        // proxyRes.headers['content-length'] = output.length
+                        delete proxyRes.headers['content-length'];
+                        delete proxyRes.headers['last-modified'];
+                        delete proxyRes.headers['max-age'];
+                        delete proxyRes.headers.etag;
+                        proxyRes.headers['cache-control'] = 'no-cache';
+                    }
+                    res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers);
+                    res.end(output);
+                }
+                catch (err) {
+                    console.error(err);
+                    res.writeHead(500);
+                    res.end(err.message);
+                }
+            });
+        }
+    }
+}));
+for (const proxyPath of config.app.proxyPaths) {
+    app.use(proxyPath, createProxyMiddleware({
+        target: appUrl.origin,
+        secure: false,
+        changeOrigin: true,
+        ws: true
+    }));
+}
+// re-expose a data-fair instance to access datasets, etc.
+const dfUrl = new URL(config.dataFair.url);
+app.use('/data-fair', createProxyMiddleware({
+    target: dfUrl.origin,
+    secure: false,
+    changeOrigin: true,
+    selfHandleResponse: true, // so that the onProxyRes takes care of sending the response
+    on: {
+        proxyReq(proxyReq, req, res) {
+            proxyReq.path = '/data-fair' + proxyReq.path;
+            // no gzip so that we can process the content
+            proxyReq.setHeader('accept-encoding', 'identity');
+            // authentication through api key only
+            proxyReq.setHeader('cookie', '');
+            if (config.dataFair.apiKey)
+                proxyReq.setHeader('x-apiKey', config.dataFair.apiKey);
+            // body was already parsed by body-parser and no longer available as a stream
+            fixRequestBody(proxyReq, req);
+        },
+        proxyRes(proxyRes, req, res) {
+            // console.log('DF RES', proxyRes)
+            if (proxyRes.headers['content-type'] && proxyRes.headers['content-type'].startsWith('application/json')) {
+                let body = '';
+                proxyRes.on('data', (data) => { body += data.toString(); });
+                proxyRes.on('end', () => {
+                    const output = body.replace(new RegExp(escapeStringRegexp(config.dataFair.url), 'g'), 'http://localhost:5888/data-fair');
+                    // proxyRes.headers['content-length'] = output.length
+                    delete proxyRes.headers['content-length'];
+                    res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers);
+                    // make all references to data-fair url point to local proxy
+                    res.end(output);
+                });
+            }
+            else {
+                res.writeHead(proxyRes.statusCode ?? 200, proxyRes.headers);
+                proxyRes.pipe(res);
+            }
+        }
+    }
+}));
+// also re-expose the simple-directory instance matching data-fair
+app.use('/simple-directory', createProxyMiddleware({
+    target: dfUrl.origin,
+    secure: false,
+    changeOrigin: true,
+    on: {
+        proxyReq(proxyReq, req, res) {
+            proxyReq.path = '/simple-directory' + proxyReq.path;
+        }
+    }
+}));
+if (config.serveUi) {
+    app.use(await createSpaMiddleware(resolve(import.meta.dirname, '../ui/dist'), uiConfig));
+}
+else {
+    app.use('/', createProxyMiddleware({
+        target: 'http://localhost:6220',
+        secure: false
+    }));
+}
+// Run app and return it in a promise
+export const run = async () => {
+    server.listen(config.port);
+    await eventPromise(server, 'listening');
+    console.log(chalk.bold.blue('\nDataFair dev server available on ') + chalk.underline.bold.blue(`http://localhost:${config.port}`));
+    return server;
+};
+export const stop = async () => {
+    if (devServerWS)
+        devServerWS.terminate();
+    if (dataFairWS)
+        dataFairWS.terminate();
+    server.close();
+    await eventPromise(server, 'close');
+    await app.get('client').close();
+};